# 导入从Boss直聘上爬取的数据
import json
from pprint import pprint
import os
import django
from tqdm import tqdm

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "JobViewer.settings")
django.setup()
from Apps.Data.models import JobModel


# 过滤虚假公司名称
def validateBrandName(brandName):
    if brandName.find('某知名') != -1 or brandName.find('某大型') != -1 or brandName.find('急聘用') != -1:
        return False
    else:
        return True


cityName = ['bj', 'gz', 'hz', 'sh', 'sz', 'wh']
folderList = ['py', 'c#', 'cpp', 'go', 'java', 'php', 'fronted']

for city in tqdm(cityName):
    for language in folderList:
        path = city + "/" + language + ".json"

        with open(path, 'r', encoding='utf-8') as file:
            json_data = file.read()
            try:
                data = json.loads(json_data)
            except:
                print(f"打开文件出错: {path}")
            print(path)
            BookedNumber = 0
            ToTal = 0

            # pass
            for item in data:
                jobList = item['zpData']['jobList']
                ToTal += len(jobList)
                for job in jobList:
                    # pprint(job)
                    cityName = job['cityName']
                    areaDistrict = job['areaDistrict']
                    brandName = job['brandName']
                    brandScaleName = job['brandScaleName']
                    jobName = job['jobName']
                    skills = ','.join(job['skills'])
                    salaryDesc = job['salaryDesc']
                    jobDegree = job['jobDegree']
                    jobExperience = job['jobExperience']
                    jobLabels = ','.join(job['jobLabels'])
                    welfareList = ','.join(job['welfareList'])

                    # 添加
                    instance = JobModel()
                    instance.set_fields(brandName, jobName,
                                        job['brandScaleName'], salaryDesc,
                                        False if job['brandStageName'] == '未融资' else True,
                                        welfareList, cityName, areaDistrict, skills, jobExperience, jobDegree,
                                        'Boss直聘',language)

                    # 数据清洗 | 判断数据是否重复/缺失
                    if not validateBrandName(brandName):
                        continue

                    try:
                        JobMatchs = JobModel.objects.filter(brand_name=brandName)
                        if not JobMatchs:
                            raise JobModel.DoesNotExist

                        for Job in JobMatchs:
                            if (Job.job_name == jobName) and (Job.city_name == cityName):
                                # print("存在相似数据")
                                BookedNumber += 1
                                break
                            else:
                                raise JobModel.DoesNotExist
                    except JobModel.DoesNotExist:
                        # 数据归档
                        instance.save()

            print(f"----- 导入成功！-----总共{ToTal}条数据，成功存储{ToTal - BookedNumber}条数据")
